Histogram Model (1D): Combination of Color and Position Features¶
Sergei Papulin (papulin.study@yandex.ru)
Contents¶
- Loading Color and Positional Elements
- Image and its Histogram
- Basic Operations in Histogram Model
- Set Operations
- Logical Operations
- Analyzing Image using Histogram Model
- Image Retrieval by Expression
- Image Retrieval by Sample
- References
Load packages¶
In [1]:
%load_ext autoreload
%autoreload 2
import sys
from plotly.subplots import make_subplots
sys.path.insert(0, "../../")
from lshist.histogram import operations
from lshist.executor import Parser, Evaluator
from lshist.utils import E
import utils
Loading Color and Positional Elements¶
Color Elements¶
Stucture of a json file with descriptions of color elements:
{
"elements": [{
"id": id,
"h": [min, max],
"s": [min, max],
"b": [min, max]},...]
}
Parameters:
id: identification of an elementh: hue, interval. If you define interval from 230 to 10, it will be divided into two ones: from 230 to 240 and from 0 to 10s: saturation, intervalb: brightness, interval
In [2]:
ELEMENT_PATH = "elements.json"
color_elements = utils.load_hist_elements_from_json(ELEMENT_PATH)
color_elements[:3]
Out[2]:
In [3]:
LOW_LEVEL_ELEMENT = "e1"
color_converter = utils.get_rgb_colors(color_elements)
print("HSB for element '{}': {}".format(LOW_LEVEL_ELEMENT, color_converter[LOW_LEVEL_ELEMENT]))
Show all colors:
In [4]:
U = list(item["id"] for item in color_elements)
colors = ["rgb{}".format(color_converter[el]) for el in U]
U_val_all = [1 for el in U]
hist_list = [U_val_all]
titles = ["Uc (all elements)"]
names = ["Uc"]
fig = utils.show_histogram(hist_list, U, colors, titles, names, main_title="Color Universal Elements")
fig.show()
Create high-level elements:
In [5]:
# Definition of high-level positional elements
Ec_green = E("e1+e2+e3+e4+e5+e6+e7+e8+e9+e10+e11+e12+e13+e14+e15+e16+e17+e18+e19+e20")
Ec_yellow_green = E("e2+e3+e21+e22+e23+e24+e25+e26+e27+e28+e29+e30")
Ec_red = E("e31+e32+e33+e34+e35+e36+e37+e38+e39+e40")
Ec_rose = E("e32+e35+e36+e39+e40")
# Sets of hight-level color elements (they will be used for the Evaluator below)
parser = Parser()
Ec_green_set = parser.parse_set(Ec_green.value)
Ec_yellow_green_set = parser.parse_set(Ec_yellow_green.value)
Ec_red_set = parser.parse_set(Ec_red.value)
Ec_rose_set = parser.parse_set(Ec_rose.value)
Show elements:
In [6]:
E1_el = Ec_green.value.strip("()").split("+")
E1_val_all = [1 if el in E1_el else 0 for el in U]
E2_el = Ec_yellow_green.value.strip("()").split("+")
E2_val_all = [1 if el in E2_el else 0 for el in U]
E3_el = Ec_red.value.strip("()").split("+")
E3_val_all = [1 if el in E3_el else 0 for el in U]
E4_el = Ec_rose.value.strip("()").split("+")
E4_val_all = [1 if el in E4_el else 0 for el in U]
hist_list = [U_val_all, E1_val_all, E2_val_all, E3_val_all, E4_val_all]
titles = ["Uc (all elements)", "Ec_green", "Ec_yellow_green", "Ec_red", "Ec_rose"]
names = ["Uc", "E1", "E2", "E3", "E4"]
fig = utils.show_histogram(hist_list, U, colors, titles, names)
fig.show()
Positional Elements¶
{
"id": id,
"pos": (x_start, y_start, width, height)
}
Create a grid for postional elements in relative units:
In [7]:
grid_1d = utils.generate_positional_grid_1d(5, 5)
grid_1d[:5]
Out[7]:
In absolute units:
In [8]:
IMAGE_WIDTH = 100
IMAGE_HEIGHT = 100
position_elements = utils.get_positional_grid_1d(IMAGE_WIDTH, IMAGE_HEIGHT, grid_1d)
position_elements[:5]
Out[8]:
In [9]:
position_converter = {el["id"]: el["pos"] for el in position_elements}
Show all positional elements:
In [10]:
img_Up = utils.generate_position_image(element="all", position_converter=position_converter)
In [11]:
fig = make_subplots(rows=1, cols=1, subplot_titles=("U"))
fig.add_image(z=img_Up, row=1, col=1, name="top")
fig.update_layout(plot_bgcolor="#fefefe", showlegend=False, height=300, width=300,
title_text="Positional Universal Elements")
fig.show()
Create high-level positional elements:
In [12]:
# Definition of high-level positional elements
Ep_top = E("e1+e2+e3+e4+e5+e6+e7+e8+e9+e10")
Ep_bottom = E("e16+e17+e18+e19+e20+e21+e22+e23+e24+e25")
Ep_left = E("e1+e2+e6+e7+e11+e12+e16+e17+e21+e22")
Ep_right = E("e4+e5+e9+e10+e14+e15+e19+e20+e24+e25")
Ep_center = E("e7+e8+e9+e12+e13+e14+e17+e18+e19")
# Sets of high-level positional elements (they will be used for the Evaluator below)
Ep_top_set = parser.parse_set(Ep_top.value)
Ep_bottom_set = parser.parse_set(Ep_bottom.value)
Ep_left_set = parser.parse_set(Ep_left.value)
Ep_right_set = parser.parse_set(Ep_right.value)
Ep_center_set = parser.parse_set(Ep_center.value)
Show elements:
In [13]:
img_bottom = utils.generate_position_image(Ep_bottom, position_converter)
img_top = utils.generate_position_image(Ep_top, position_converter)
img_left = utils.generate_position_image(Ep_left, position_converter)
img_right = utils.generate_position_image(Ep_right, position_converter)
img_center = utils.generate_position_image(Ep_center,position_converter)
In [14]:
fig = make_subplots(rows=1, cols=5, subplot_titles=("Top", "Bottom", "Left", "Right", "Center"))
fig.add_image(z=img_top, row=1, col=1, name="top")
fig.add_image(z=img_bottom, row=1, col=2, name="bottom")
fig.add_image(z=img_left, row=1, col=3, name="left")
fig.add_image(z=img_right, row=1, col=4, name="right")
fig.add_image(z=img_center, row=1, col=5, name="center")
fig.update_layout(plot_bgcolor="#fefefe", showlegend=True, height=300, width=900, title_text="Positional Elements")
fig.show()
Image and its Histogram¶
Generate an image composed from elements of the universal set:
In [15]:
Uc = list(item["id"] for item in color_elements)
In [16]:
img = utils.generate_image(Uc, color_converter, delta=10, add_normal_color=["e33", "e34"], seed=1)
img
Out[16]:
Create a histogram from the image
In [17]:
hist = utils.convert2hist_1d(img, color_elements, grid_1d)
Showing Image and its Histogram
In [18]:
hist_elements = sorted(hist.hist_elements().items(), key=lambda x: int(x[0][1].strip("e")))
elements = ["({})".format(",".join(el[0])) for el in hist_elements]
values = [el[1].value for el in hist_elements]
colors = ["rgb{}".format(color_converter[el[0][1]]) for el in hist_elements]
In [19]:
fig = make_subplots(rows=1, cols=2, column_widths=[0.2, 0.8], subplot_titles=("Image", "Histogram"))
fig.add_image(z=img, row=1, col=1, name="image")
fig.add_bar(x=elements, y=values, marker_color=colors, width=0.5, row=1, col=2, name="histogram")
fig.update_xaxes(gridcolor='#bdbdbd', title="Elements", titlefont=dict(color="grey"), row=1, col=2)
fig.update_yaxes(gridcolor='#bdbdbd', title="Counts", titlefont=dict(color="grey"), row=1, col=2)
fig.update_layout(plot_bgcolor='#fefefe', showlegend=False, height=300, width=900, title_text="Initial Data")
fig.show()
Sum of all histogram element values:
In [20]:
hist.sum()
Out[20]:
Get a value of the (e8,e2) element:
In [21]:
hist(("e8","e2")).sum()
Out[21]:
In [22]:
len(hist)
Out[22]:
Show the high-level positional elements in the context of the image:
In [23]:
img_context_bottom = utils.generate_position_image_with_context(img, Ep_bottom_set, position_converter)
img_context_top = utils.generate_position_image_with_context(img, Ep_top_set, position_converter)
img_context_left = utils.generate_position_image_with_context(img, Ep_left_set, position_converter)
img_context_right = utils.generate_position_image_with_context(img, Ep_right_set, position_converter)
img_context_center = utils.generate_position_image_with_context(img, Ep_center_set, position_converter)
In [24]:
fig = make_subplots(rows=1, cols=5, subplot_titles=("Top", "Bottom", "Left", "Right", "Center"))
fig.add_image(z=img_context_top, row=1, col=1, name="top")
fig.add_image(z=img_context_bottom, row=1, col=2, name="bottom")
fig.add_image(z=img_context_left, row=1, col=3, name="left")
fig.add_image(z=img_context_right, row=1, col=4, name="right")
fig.add_image(z=img_context_center, row=1, col=5, name="center")
fig.update_layout(plot_bgcolor="#fefefe", showlegend=True, height=300, width=800, title_text="Positional Elements")
fig.show()
Basic Operations in Histogram Model¶
Initialize a dictionary of high-level elements:
In [25]:
high_level_elements = {
0: { # position
"Ep_top": Ep_top_set,
"Ep_bottom": Ep_bottom_set,
"Ep_left": Ep_left_set,
"Ep_right": Ep_right_set,
"Ep_center": Ep_center_set
},
1: { # color
"Ec_red": Ec_red_set,
"Ec_green": Ec_green_set
}
}
Initialize the parser and evaluator:
In [26]:
parser = Parser()
evaluator = Evaluator(operations, hist, high_level_elements=high_level_elements)
Define high-level elements:
In [27]:
E_top_red = E("Ep_top, Ec_red")
E_bottom_red = E("Ep_bottom, Ec_red")
E_left_red = E("Ep_left, Ec_red")
E_right_red = E("Ep_right, Ec_red")
E_center_red = E("Ep_center, Ec_red")
Histograms of E_top_red, E_bottom_red, E_left_red, E_right_red and E_center_red given Image¶
Create a histogram of E_top_red given the image:
In [28]:
E_top_red_expr = parser.parse_string(E_top_red.value)
HE_top_red = evaluator.eval(E_top_red_expr)
print("Expression for E_top_red:\n{}".format(E_top_red.value))
print("\nThe parsed expressino for E_top_red in the postfix notation:\n{}".format(E_top_red_expr))
print("\nHistogram of E_top_red given the image:\n{}".format(HE_top_red.to_dict()))
print("\nValue of presence for E_union:\n{}".format(HE_top_red.sum()))
Same for E_bottom_red, E_left_red, E_right_red and E_center_red:
In [29]:
E_bottom_red_expr = parser.parse_string(E_bottom_red.value)
E_left_red_expr = parser.parse_string(E_left_red.value)
E_right_red_expr = parser.parse_string(E_right_red.value)
E_center_red_expr = parser.parse_string(E_center_red.value)
HE_bottom_red = evaluator.eval(E_bottom_red_expr)
HE_left_red = evaluator.eval(E_left_red_expr)
HE_right_red = evaluator.eval(E_right_red_expr)
HE_center_red = evaluator.eval(E_center_red_expr)
Show the histograms of E_top_red, E_bottom_red, E_left_red, E_right_red and E_center_red given the image:
In [30]:
HE_list = [HE_top_red, HE_bottom_red, HE_left_red, HE_right_red, HE_center_red]
image_titles = ["(Top, Red)", "(Bottom, Red)", "(Left, Red)", "(Right, Red)", "(Center, Red)"]
hist_titles = ["E_top_red", "E_bottom_red", "E_left_red", "E_right_red", "E_center_red"]
fig = utils.show_histogram1d(HE_list=HE_list, img=img, color_converter=color_converter,
position_converter=position_converter,
image_titles=image_titles, hist_titles=hist_titles)
fig.show()
Set Operations¶
In [31]:
E1 = E("Ep_top, Ec_red")
E2 = E("Ep_right, Ec_red")
In [32]:
E1_expr = parser.parse_string(E1.value)
E2_expr = parser.parse_string(E2.value)
HE1 = evaluator.eval(E1_expr)
HE2 = evaluator.eval(E2_expr)
UNION¶
In [33]:
E_union = E1 + E2
E_union_expr = parser.parse_string(E_union.value)
HE_union = evaluator.eval(E_union_expr)
print("Expression for E_union:\n{}".format(E_union))
print("\nThe parsed expression for E_union in the postfix notation:\n{}".format(E_union_expr))
print("\nHistogram of E_union given the image:\n{}".format(HE_union.to_dict()))
print("\nValue of presence for E_union:\n{}".format(HE_union.sum()))
Show the histogram of E_union given the image:
In [34]:
HE_list = [HE1, HE2, HE_union]
image_titles = ["(Top, Red)", "(Right, Red)", "(Top, Red) + (Right, Red)"]
hist_titles = ["E1", "E2", "E1 + E2"]
fig = utils.show_histogram1d(HE_list=HE_list, img=img, color_converter=color_converter,
position_converter=position_converter,
image_titles=image_titles, hist_titles=hist_titles)
fig.show()
INTERSECTION¶
In [35]:
E_intersect = E1 * E2 # or E1.Intersection(E2)
E_intersect_expr = parser.parse_string(E_intersect.value)
HE_intersect = evaluator.eval(E_intersect_expr)
print("Expression for E_intersect:\n{}".format(E_intersect))
print("\nThe parsed expression for E_intersect in the postfix notation:\n{}".format(E_intersect_expr))
print("\nHistogram of E_intersect given the image:\n{}".format(HE_intersect.to_dict()))
print("\nValue of presence for E_intersect:\n{}".format(HE_intersect.sum()))
Show the histogram of E_intersect given the image:
In [36]:
HE_list = [HE1, HE2, HE_intersect]
image_titles = ["(Top, Red)", "(Right, Red)", "(Top, Red) * (Right, Red)"]
hist_titles = ["E1", "E2", "E1 * E2"]
fig = utils.show_histogram1d(HE_list=HE_list, img=img, color_converter=color_converter,
position_converter=position_converter,
image_titles=image_titles, hist_titles=hist_titles)
fig.show()
SUBSTRACTION or EXCEPTION¶
In [37]:
E_sub = E1 - E2 # or E1.Sub(E2)
E_sub_expr = parser.parse_string(E_sub.value)
HE_sub = evaluator.eval(E_sub_expr)
print("Expression for E_sub:\n{}".format(E_sub))
print("\nThe parsed expression for E_sub in the postfix notation:\n{}".format(E_sub_expr))
print("\nHistogram of E_sub given the image:\n{}".format(HE_sub.to_dict()))
print("\nValue of presence for E_sub:\n{}".format(HE_sub.sum()))
Show the histogram of E_sub given the image:
In [38]:
HE_list = [HE1, HE2, HE_sub]
image_titles = ["(Top, Red)", "(Right, Red)", "(Top, Red) - (Right, Red)"]
hist_titles = ["E1", "E2", "E1 - E2"]
fig = utils.show_histogram1d(HE_list=HE_list, img=img, color_converter=color_converter,
position_converter=position_converter,
image_titles=image_titles, hist_titles=hist_titles)
fig.show()
Logical Operations¶
AND¶
In [39]:
E_and = E1 & E2 # or E1.And(E2)
E_and_expr = parser.parse_string(E_and.value)
HE_and = evaluator.eval(E_and_expr)
print("Expression for E_and:\n{}".format(E_and))
print("\nThe parsed expression for E_and in the postfix notation:\n{}".format(E_and_expr))
print("\nHistogram of E_and given the image:\n{}".format(HE_and.to_dict()))
print("\nValue of presence for E_and:\n{}".format(HE_and.sum()))
Show the histogram of E_and given the image:
In [40]:
HE_list = [HE1, HE2, HE_and]
image_titles = ["(Top, Red)", "(Right, Red)", "(Top, Red) <b>AND</b> (Right, Red)"]
hist_titles = ["E1", "E2", "E1 <b>AND</b> E2"]
fig = utils.show_histogram1d(HE_list=HE_list, img=img, color_converter=color_converter,
position_converter=position_converter,
image_titles=image_titles, hist_titles=hist_titles)
fig.show()
OR¶
In [41]:
E_or = E1 | E2 # or E1.Or(E2)
E_or_expr = parser.parse_string(E_or.value)
HE_or = evaluator.eval(E_or_expr)
print("Expression for E_or:\n{}".format(E_or))
print("\nThe parsed expression for E_or in the postfix notation:\n{}".format(E_or_expr))
print("\nHistogram of E_or given the image:\n{}".format(HE_or.to_dict()))
print("\nValue of presence for E_or:\n{}".format(HE_or.sum()))
Show the histogram of E_or given the image:
In [42]:
HE_list = [HE1, HE2, HE_or]
image_titles = ["(Top, Red)", "(Right, Red)", "(Top, Red) <b>OR</b> (Right, Red)"]
hist_titles = ["E1", "E2", "E1 <b>OR</b> E2"]
fig = utils.show_histogram1d(HE_list=HE_list, img=img, color_converter=color_converter,
position_converter=position_converter,
image_titles=image_titles, hist_titles=hist_titles)
fig.show()
XOR¶
In [43]:
E_xor = E1 ^ E2 # or E1.Xor(E2)
E_xor_expr = parser.parse_string(E_xor.value)
HE_xor = evaluator.eval(E_xor_expr)
print("Expression for E_xor:\n{}".format(E_xor))
print("\nThe parsed expression for E_xor in the postfix notation:\n{}".format(E_xor_expr))
print("\nHistogram of E_xor given the image:\n{}".format(HE_xor.to_dict()))
print("\nValue of presence for E_xor:\n{}".format(HE_xor.sum()))
Show the histogram of E_xor given the image:
In [44]:
HE_list = [HE1, HE2, HE_xor]
image_titles = ["(Top, Red)", "(Right, Red)", "(Top, Red) <b>XOR</b> (Right, Red)"]
hist_titles = ["E1", "E2", "E1 <b>XOR</b> E2"]
fig = utils.show_histogram1d(HE_list=HE_list, img=img, color_converter=color_converter,
position_converter=position_converter,
image_titles=image_titles, hist_titles=hist_titles)
fig.show()
XSUBSTRACTION¶
Case 1
In [45]:
E_xsub = E1.Xsub(E2)
E_xsub_expr = parser.parse_string(E_xsub.value)
HE_xsub = evaluator.eval(E_xsub_expr)
print("Expression for E_xsub:\n{}".format(E_xsub))
print("\nThe parsed expression for E_xsub in the postfix notation:\n{}".format(E_xsub_expr))
print("\nHistogram of E_xsub given the image:\n{}".format(HE_xsub.to_dict()))
print("\nValue of presence for E_xsub:\n{}".format(HE_xsub.sum()))
Show the histogram of E_xsub given the image:
In [46]:
HE_list = [HE1, HE2, HE_xsub]
image_titles = ["(Top, Red)", "(Right, Red)", "(Top, Red) <b>XSUBSTRACT</b> (Right, Red)"]
hist_titles = ["E1", "E2", "E1 <b>XSUBSTRACT</b> E2"]
fig = utils.show_histogram1d(HE_list=HE_list, img=img, color_converter=color_converter,
position_converter=position_converter,
image_titles=image_titles, hist_titles=hist_titles)
fig.show()
Case 2
In [47]:
E3 = E("Ep_right, e22")
E3_expr = parser.parse_string(E3.value)
HE3 = evaluator.eval(E3_expr)
In [48]:
E_xsub = E1.Xsub(E3)
E_xsub_expr = parser.parse_string(E_xsub.value)
HE_xsub = evaluator.eval(E_xsub_expr)
print("Expression for E_xsub:\n{}".format(E_xsub))
print("\nThe parsed expression for E_xsub in the postfix notation:\n{}".format(E_xsub_expr))
print("\nHistogram of E_xsub given the image:\n{}".format(HE_xsub.to_dict()))
print("\nValue of presence for E_xsub:\n{}".format(HE_xsub.sum()))
In [49]:
HE_list = [HE1, HE3, HE_xsub]
image_titles = ["(Top, Red)", "(Right, e22)", "(Top, Red) <b>XSUBSTRACT</b> (Right, e22)"]
hist_titles = ["E1", "E3", "E1 <b>XSUBSTRACT</b> E3"]
fig = utils.show_histogram1d(HE_list=HE_list, img=img, color_converter=color_converter,
position_converter=position_converter,
image_titles=image_titles, hist_titles=hist_titles)
fig.show()
Analyzing Image using Histogram Model¶
In [50]:
# TOD0
'''
E1 = E("all, e23")
E2 = E("e5, e12")
expression = parser.parse_string((E1+E2).value)
HE_result = evaluator.eval(expression)
HE_result.to_dict()
'''
Out[50]:
Image Retrieval by Expression¶
Generate images:
In [51]:
imgs = [utils.generate_image(Uc, color_converter, delta=10, add_normal_color=["e33", "e34"], seed=i)
for i in range(100)]
imgs += [utils.generate_image(Uc, color_converter, delta=10, seed=i) for i in range(100)]
Create histograms for the images:
In [52]:
hists = [utils.convert2hist_1d(img, color_elements, grid_1d) for img in imgs]
Compose a query to get a subset of relevant images:
In [53]:
query = E1 + E2
expr = parser.parse_string(query.value)
Calculate histograms of the query for the images:
In [54]:
HEs = [evaluator.eval(expr, hist) for hist in hists]
Rank the images based on their values of presence:
In [55]:
img_rank = sorted([(indx, HE.sum()) for indx, HE in enumerate(HEs)], key=lambda x: -x[1])
print("Top 5 images:", img_rank[:5])
print("Last 5 images:", img_rank[-5:])
Image that most corresponds to the query:
In [56]:
imgs[img_rank[0][0]]
Out[56]:
Show the top 5 images:
In [57]:
top_5_imgs = [imgs[rank[0]] for rank in img_rank[:5]]
fig = utils.show_rank_images(top_5_imgs, "Top 5")
fig.show()
Image that least corresponds to the query:
In [58]:
imgs[img_rank[len(imgs)-1][0]]
Out[58]:
Show the last 5 images:
In [59]:
last_5_imgs = [imgs[rank[0]] for rank in img_rank[-5:]]
fig = utils.show_rank_images(last_5_imgs, "Last 5")
fig.show()
Image Retrieval by Sample¶
Generate a sample image:
In [60]:
img_sample = utils.generate_image(Uc, color_converter, delta=10, add_normal_color=["e33", "e34"], seed=300)
img_sample
Out[60]:
Create the histogram of the image:
In [61]:
hist_sample = utils.convert2hist_1d(img_sample, color_elements, grid_1d)
Rank the images according to their similarity:
In [62]:
img_sample_rank = sorted([(indx, (hist_sample * hist).sum()) for indx, hist in enumerate(hists)],
key=lambda x: -x[1])
print("Top 5 images:", img_sample_rank[:5])
print("Last 5 images:", img_sample_rank[-5:])
Show the top 5 images:
In [63]:
top_5_imgs = [imgs[rank[0]] for rank in img_sample_rank[:5]]
fig = utils.show_rank_images(top_5_imgs, "Top 5")
fig.show()
Show the last 5 images:
In [64]:
last_5_imgs = [imgs[rank[0]] for rank in img_sample_rank[-5:]]
fig = utils.show_rank_images(last_5_imgs, "Last 5")
fig.show()